﻿import MySQLdb
import funcs


class schedule(object):
    def __init__(self):
        self.cxn=MySQLdb.connect(db='twi_pachong',host='localhost',user='root',passwd='123456ms')
        self.cxn.set_character_set('utf8')
        self.cur=self.cxn.cursor()
        #前12位为段地址，11位为段偏移地址，最后5位为整型地址
        self.url_set=[0]*(2**12)
        for i in xrange(len(self.url_set)):
            self.url_set[i]=[0]*(2**11)
            
    def get_table_data(self,table_name):
        print '    正在读取%s表'%table_name
        sql='select count(*) from %s'%table_name
        self.cur.execute(sql)
        count=self.cur.fetchone()[0]
        if count>0:
            sql='select max(id) from %s'%table_name
            self.cur.execute(sql)
            max_id=self.cur.fetchone()[0]
            
            sql='select min(id) from %s'%table_name
            self.cur.execute(sql)
            min_id=self.cur.fetchone()[0]
            print min_id,max_id
            step=10**8
            idx=min_id
            num=0
            while idx<=max_id:
                sql_pref='select id from %s'%table_name
                sql=sql_pref+' where id between %s and %s'
                self.cur.execute(sql,(str(idx),str(idx+step)))
                for record in self.cur.fetchall():
                    _id=record[0]
                    i_md5=int(_id)
                    if self.set_url_mark(i_md5)==False:
                        num+=1
                        print num
                idx+=step
        print '    %s表读取完毕'%table_name
        
    def build_url_set(self):
        print '正在建立url_set'
        #前12位为段地址，11位为段偏移地址，最后5位为整型地址
        self.url_set=[0]*(2**12)
        for i in xrange(len(self.url_set)):
            self.url_set[i]=[0]*(2**11)

        table_list=['self_url','url_queue','self_url']
        for table in table_list:
            self.get_table_data(table)
        
        print 'url_set建立完毕'

    def set_url_mark(self,l_md5):
        seg_index,seg_offset,offset_value=funcs.get_bit_size(l_md5)
        u_offset_value=self.url_set[seg_index][seg_offset]
        b_u_o_b=(u_offset_value>>offset_value)%2
        if b_u_o_b==0:
            u_offset_value=u_offset_value|(1<<offset_value)
            self.url_set[seg_index][seg_offset]=u_offset_value
            return_value=True
        else:
            return_value=False
        return return_value
if __name__=='__main__':
    sche=schedule()
    table_list=['self_url','url_queue']

    sche.build_url_set()
    











